# -*- coding: utf-8 -*-
"""
Created on Mon Feb 22 09:19:06 2021
Modified on 2021年4月1日17:15:02
@author: 59567
@author: Qingtian

Changelog:
    2021年4月1日，叶擎天：
        1.“保存特征集”直接保存到x_in_set_selector.xlsx
        2.“搜索”列表输出变成换行输出，可以直接复制粘贴到excel

Usage：
    特征选择与排除：
        1.Source下拉选择x数据集
        2.key键入搜索关键字，如“库存”，并点击“search”
        2.1 使用“Find Similar”搜索与该特征相似的其他特征
        A.直接复制到exccel：
            A.1.在左下框中复制特征，用excel打开x_out_set_selector或x_in_set_selector，粘贴到你的y_name下面
        B.先加入购物车再导出到x_in_set_selector特征：
            B.1.在Select下拉框中选择需要添加的特征后“加入特征集”
            B.2.选够需要添加的特征后，“保存到”
            B.3.弹出对话框中选择你的y_name
            B.4.成功有弹窗提示。

"""

import sys
import numpy as np
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2Tk
from matplotlib.backend_bases import key_press_handler
import matplotlib.pyplot as plt
import tkinter as tk
from tkinter import Frame, Label, Text, StringVar, Button, END, Entry, simpledialog
from tkinter.ttk import Combobox, Spinbox
from copy import deepcopy
from tjdutils.utils import real_path
from tjdutils.fractal_dimension import calculate_fractal_dimension as cal_fd
from pandas import read_pickle, date_range, DataFrame, read_excel, Series, to_datetime
from matplotlib.figure import Figure

plt.rcParams['font.sans-serif'] = ['SimHei']  # 中文显示
plt.rcParams['axes.unicode_minus'] = False  # 负号显示
from sklearn.metrics import mean_absolute_error as mae
from tjdutils.files_operation import input_x_pkls
from mpl_toolkits.mplot3d import Axes3D
from sklearn import decomposition, preprocessing, linear_model, metrics, manifold
from pandas import Timestamp, concat
from data.ini_data import ini_all_x_data
sys.path.append('../')


# ------------------------------------------------------------------------------------------

class From():
    def __init__(self):
        super(From, self).__init__()
        self.root = tk.Tk()  # 创建主窗体
        self.root.title("Welcome to use TJD high frequence economic monitor systerm")
        self.root.geometry("1850x1000")
        self.bg = "CornflowerBlue"  # 设置背景色
        self.m_f = ('Microsoft YaHei', 20,)  # main_font, 主要字体
        self.s_f = ('Microsoft YaHei', 15,)  # second_font, 二级字体
        self.t_f = ('Microsoft YaHei', 12,)  # third_font, 三级字体
        self.canvas = tk.Canvas()  # 创建一块显示图形的画布
        self.tool_ready = False

        # self.y = read_excel(f'input/y_plus_集合.xlsx', index_col=0)
        self.raw_x = read_pickle(real_path(input_x_pkls()[-1]))
        self.all_features = self.raw_x.columns.to_list()
        self.summary = None
        self.hf_df = None
        self.correct_hf_df = None
        self.select_feature = []
        self.print_feature = 0
        self.canvas_select = None
        self.end_datetimes = [1, 2, 3]
        self.cart = {}

        self.fm_args = Frame(self.root, height=520, width=390, relief='ridge', bd=5, bg=self.bg)
        self.fm_args.place(x=10, y=10)
        self.fm_print = Frame(self.root, height=400, width=390, relief='ridge', bd=5, bg=self.bg)
        self.fm_print.place(x=10, y=540)
        self.fm_show1 = Frame(self.root, height=480, width=860, relief='ridge', bd=5, bg=self.bg)
        self.fm_show1.place(x=415, y=10)
        self.fm_show2 = Frame(self.root, height=480, width=860, relief='ridge', bd=5, bg=self.bg)
        self.fm_show2.place(x=415, y=500)
        # ----------------------------------------label----------------------
        self.label_args = {
            'key': {'master': self.fm_args, 'text': 'Key:', 'font': self.t_f, },
            'select_key': {'master': self.fm_args, 'text': 'Select:', 'font': self.t_f, },
            'freq': {'master': self.fm_args, 'text': 'Freq:', 'font': self.t_f, },
            'source': {'master': self.fm_args, 'text': 'X数据集:', 'font': self.t_f},
            'reduce_method': {'master': self.fm_args, 'text': '降维方法:', 'font': self.t_f, },
            'y_select': {'master': self.fm_args, 'text': '选择Y:', 'font': self.t_f, },
            'frag_dimension': {'master': self.fm_args, 'text': '分数维度:', 'font': self.t_f, },
        }
        self.label_place_args = {
            'key': {'x': 10, 'y': 20},
            'select_key': {'x': 2, 'y': 80},
            'freq': {'x': 10, 'y': 180},
            'source': {'x': 5, 'y': 270},
            'reduce_method': {'x': 5, 'y': 420},
            'y_select': {'x': 5, 'y': 450},
            'frag_dimension': {'x': 5, 'y': 480},
        }
        # ----------------------------------------inputs----------------------
        self.input_args = {
            'key': {'master': self.fm_args, 'font': self.t_f, 'width': 25},
        }
        self.input_place_args = {
            'key': {'x': 60, 'y': 20},
        }
        # ----------------------------------------texts-----------------------
        self.text_args = {
            'speech': {'master': self.fm_print, 'font': self.t_f, 'width': 40, 'height': 15},
        }
        self.text_place_args = {
            'speech': {'x': 15, 'y': 10},
        }
        self.text_insert_args = {
            'speech': {'index': 1.0,
                       'chars': """
    特征筛选：
    1.Source下拉选择x数据集
    2.key键入搜索关键字，如“库存”，并点击“search”
    2.1 (可选)使用“Find Similar”搜索与该特征相似的其他特征
    A.直接复制到exccel：
        A.1.在左下框中复制特征，用excel打开x_out_set_selector或x_in_set_selector，粘贴到你的y_name下面
    B.先加入购物车再导出到x_in_set_selector特征：
        B.1.在Select下拉框中选择需要添加的特征后“加入特征集”
        B.2.选够需要添加的特征后，“保存特征集到x_in_set”
        B.3.弹出对话框中选择你的y_name
        B.4.成功有弹窗提示。"""
                       },
        }
        # ----------------------------------------boxs-------------------------
        self.most_box_args = {'master': self.fm_args, 'width': 9, }
        self.box_args = {
            'select_feature_box': {'master': self.fm_args, 'width': 30, },
            'freq': {'master': self.fm_args, 'width': 8, },
            'source': {'master': self.fm_args, 'width': 30, },
            'reduce_method': {'master': self.fm_args, 'width': 22, },
            'y_select': {'master': self.fm_args, 'width': 22, },
        }
        self.box_value_args = {
            'select_feature_box': ['please type a key and search', '..', '..'],
            'freq': ['D', 'M'],
            'source': input_x_pkls(),
            'reduce_method': ['PCA',
                              'KernelPCA_sigmoid', 'KernelPCA_rbf', 'KernelPCA_poly',
                              'TruncatedSVD',
                              'FastICA', 'Isomap'
                              ],
            'y_select': ['-', '-'],
            # sorted(list(self.y.columns)),
        }
        self.box_value_type_args = ['period1', 'period2', 'period3']
        self.box_place_args = {
            'select_feature_box': {'x': 60, 'y': 80},
            'freq': {'x': 60, 'y': 180},
            'source': {'x': 80, 'y': 270},
            'reduce_method': {'x': 80, 'y': 420},
            'y_select': {'x': 80, 'y': 450},
        }
        # ----------------------------------------button-----------------------
        self.button_args = {
            'search': {'master': self.fm_args, 'text': 'Search', 'font': self.t_f, 'width': 6},
            'plot1': {'master': self.fm_args, 'text': 'Add1', 'font': self.t_f, 'width': 6},
            'plot2': {'master': self.fm_args, 'text': 'Add2', 'font': self.t_f, 'width': 6},
            'last': {'master': self.fm_args, 'text': 'Last', 'font': self.t_f, 'width': 6, 'height': 1},
            'next': {'master': self.fm_args, 'text': 'Next', 'font': self.t_f, 'width': 6, 'height': 1},
            'match': {'master': self.fm_args, 'text': 'Find Similar', 'font': self.t_f, 'width': 9, 'height': 1},
            'add_cart': {'master': self.fm_args, 'text': '加入特征集', 'font': self.t_f, 'width': 16, 'height': 1},
            'save_cart': {'master': self.fm_args, 'text': '保存特征集到x_in_set', 'font': self.t_f, 'width': 16, 'height': 1},
            'reduce': {'master': self.fm_args, 'text': '降维并绘图', 'font': self.t_f, 'width': 9, 'height': 1},

        }
        self.button_place_args = {
            'search': {'x': 300, 'y': 15},
            'plot1': {'x': 100, 'y': 120},
            'plot2': {'x': 200, 'y': 120},
            'last': {'x': 300, 'y': 70},
            'next': {'x': 300, 'y': 110},
            'match': {'x': 270, 'y': 180},
            'add_cart': {'x': 10, 'y': 220},
            'save_cart': {'x': 200, 'y': 220},
            'reduce': {'x': 270, 'y': 420},

        }
        self.button_command_args = {
            'search': self._search,
            'plot1': self._plot1,
            'plot2': self._plot2,
            'last': self._last_plot,
            'next': self._next_plot,
            'match': self._find_similar,
            'add_cart': self._add_cart,
            'save_cart': self._save_cart,
            'reduce': self.reduce,

        }
        # ----------------------------------------spin-----------------------
        self.spin_args = {
            'frag_number': {'master': self.fm_args, 'values': list(range(5, 25)), 'width': 10},
            'end_datetime': {'master': self.fm_args,
                             'values': [str(t)[:10] for t in date_range('2008-12-31', '2020-12-31', freq='M')],
                             'width': 10},
        }
        self.spin_command_args = {
            'frag_number': self._fractal_dimension,
            'end_datetime': self._fractal_dimension,
        }
        self.spin_place_args = {
            'frag_number': {'x': 80, 'y': 480},
            'end_datetime': {'x': 200, 'y': 480},
        }
        self._ini_all()

    def _on_key_press1(self, event):
        print("you pressed {}".format(event.key))
        key_press_handler(event, self.canvas1, self.toolbar1)

    def _on_key_press2(self, event):
        print("you pressed {}".format(event.key))
        key_press_handler(event, self.canvas2, self.toolbar2)

    def _ini_all(self):
        # --------------------------ini---input--box---button---box-------------
        self.all_labels = self._label_ini()
        self.all_inputs = self._input_ini()
        self.all_boxs = self._box_ini()
        self.all_texts = self._text_ini()
        self.all_buttons = self._button_ini()
        self.all_spins = self._spin_ini()
        # ----------------------------------------初始化图像--------------------
        self.fig1 = Figure(figsize=(14, 4), dpi=100)
        self.canvas1 = FigureCanvasTkAgg(self.fig1, self.fm_show1)
        self.canvas1.get_tk_widget().pack(side=tk.TOP, fill=tk.BOTH, expand=1)
        self.toolbar1 = NavigationToolbar2Tk(self.canvas1, self.fm_show1)
        self.toolbar1.update()
        self.canvas1.mpl_connect("key_press_event", self._on_key_press1)

        self.fig2 = Figure(figsize=(14, 4), dpi=100)
        self.canvas2 = FigureCanvasTkAgg(self.fig2, self.fm_show2)
        self.canvas2.get_tk_widget().pack(side=tk.TOP, fill=tk.BOTH, expand=1)
        self.toolbar2 = NavigationToolbar2Tk(self.canvas2, self.fm_show2)
        self.toolbar2.update()
        self.canvas2.mpl_connect("key_press_event", self._on_key_press2)

        self.all_boxs['freq'].current(0)
        self.x = self._process_freq()

    def _label_ini(self, all_labes={}):
        for k, v in self.label_place_args.items():
            all_labes[k] = Label(**self.label_args[k], bg=self.bg, )  # 选好字段
            all_labes[k].place(**v)  # 放位置

    def _input_ini(self, all_inputs={}):
        for k, v in self.input_place_args.items():
            all_inputs[k] = Entry(**self.input_args[k])
            all_inputs[k].place(**v)
        return all_inputs

    def _text_ini(self, all_texts={}):
        for k, v in self.text_place_args.items():
            all_texts[k] = Text(**self.text_args[k])
            all_texts[k].place(**v)
            all_texts[k].insert(**self.text_insert_args[k])
        return all_texts

    def _box_ini(self, all_box={}, all_v={}):
        for k, v in self.box_place_args.items():
            all_v[k] = StringVar()
            all_box[k] = Combobox(**self.box_args[k], textvariable=all_v[k])  # 选好字段
            all_box[k]['values'] = self.box_value_args[k]
            all_box[k].place(**v)  # 放位置
        all_box['freq'].bind("<<ComboboxSelected>>", self._freq)
        all_box['source'].bind("<<ComboboxSelected>>", self._set_raw_x)
        return all_box

    def _button_ini(self, all_buttons={}):
        for k, v in self.button_place_args.items():
            print(k)
            if k in list(self.button_args.keys()):
                all_buttons[k] = Button(**self.button_args[k], command=self.button_command_args[k])
            else:
                all_buttons[k] = Button(**self.most_button_args, command=self.button_command_args[k])
            all_buttons[k].place(**v)
        return all_buttons

    def _spin_ini(self, all_spins={}):
        for k, v in self.spin_place_args.items():
            all_spins[k] = Spinbox(**self.spin_args[k], command=self.spin_command_args[k])
            all_spins[k].place(**v)
        return all_spins

    def _process_freq(self):
        self.x = deepcopy(self.raw_x)
        freq = self.all_boxs['freq'].get()
        if freq == 'M':
            dtr = date_range(start=self.x.index[0], end=self.x.index[-1], freq='M')
            if str(dtr[-1]) not in self.x.index:
                dtr = dtr[:-1]
            dtr = [str(t)[:10] for t in dtr]
            self.x = self.x.loc[dtr, :]
        return self.x

    def _freq(self, event):
        self._process_freq()

    def _set_raw_x(self, event):
        path = self.all_boxs['source'].get()
        print(path)
        self.raw_x = read_pickle(real_path(path))
        self.x = deepcopy(self.raw_x)
        self.all_features = self.raw_x.columns.to_list()

    def _search(self):
        keyword = self.all_inputs['key'].get()
        print(keyword)
        if '&' in keyword:
            keywords = keyword.split('&')
            self.select_feature = [f for f in self.all_features if keywords[0] in f]
            self.select_feature = [f for f in self.select_feature if keywords[1] in f]
            self.all_boxs['select_feature_box']['values'] = self.select_feature
            self._update_text('speech', "\n".join(self.select_feature))
        else:
            self.select_feature = [feature for feature in self.all_features if keyword in feature]
            self.all_boxs['select_feature_box']['values'] = self.select_feature
            self._update_text('speech', "\n".join(self.select_feature))

    def _update_text(self, text, s):
        self.all_texts[text].delete(1.0, END)
        self.all_texts[text].insert(1.0, s)

    def _ax_plot(self, fig):
        fig.clear()
        ax = fig.add_subplot(111)
        feature = self.all_boxs['select_feature_box'].get()
        end_datetime = self.all_spins['end_datetime'].get()
        self.print_feature = self.select_feature.index(feature)
        xs = self.x.loc[:, feature]
        ax.plot(xs, label=feature)
        if len(end_datetime) > 1:
            ax.axvline(x=to_datetime(end_datetime), ls='--', lw=1, c='green')
            ax.text(to_datetime(end_datetime), max(xs), end_datetime)
        ax.legend(loc='upper left')

    def _plot1(self):
        self._ax_plot(self.fig1)
        self.canvas1.draw()
        self.canvas_select = 1

    def _plot2(self):
        self._ax_plot(self.fig2)
        self.canvas2.draw()
        self.canvas_select = 2

    def auto_plot(self):
        if self.canvas_select == 1:
            self._plot1()
        else:
            self._plot2()

    def _last_plot(self):
        self.print_feature -= 1
        self.all_boxs['select_feature_box'].current(self.print_feature)
        self.auto_plot()

    def _next_plot(self):
        self.print_feature += 1
        self.all_boxs['select_feature_box'].current(self.print_feature)
        self.auto_plot()

    def _find_similar(self):
        feature = self.all_boxs['select_feature_box'].get()
        mae_df = DataFrame(data=np.nan, columns=['mae'], index=self.all_features)
        for c in self.all_features:
            mae_df.loc[c, 'mae'] = mae(self.x.loc[:, feature], self.x.loc[:, c])
        mae_df.sort_values(by='mae', ascending=True, inplace=True)
        long_s = ''
        for ss in mae_df.index[:20]:
            long_s = long_s + ss + '\\'
        self._update_text('speech', long_s)

    def _add_cart(self):
        feature = self.all_boxs['select_feature_box'].get()
        xs = self.x.loc[:, feature]
        self.cart[feature] = xs
        print(feature)
        s = feature + '已加入特征集合' + '当前已经有特征' + str(len(self.cart)) + '个'
        self._update_text('speech', s)

    def _save_cart(self):
        import pandas as pd
        self.x_in = pd.read_excel(real_path("x_in_set_selector.xlsx"))
        self.x_in = self.x_in.reindex(range(0, self.x_in.index[-1] * 2 + 500))
        self.cart_features = list(self.cart.keys())
        self.cart_features_len = len(self.cart_features)
        save_to_y = self.askComboValue(*self.x_in.columns.to_list())
        print(f"保存所选特征到{save_to_y}")

        for index, value in self.x_in[save_to_y].iteritems():
            if len(self.cart_features) == 0:
                self.x_in.dropna(axis=0, how="all", inplace=True)
                self.x_in.to_excel(real_path("x_in_set_selector.xlsx"), index=False)
                tk.messagebox.showinfo(title="成功", message=f"成功将{self.cart_features_len}个特征保存到{save_to_y}")
                break
            if pd.isna(value):
                cart_features_temp = self.cart_features.pop()
                print(f"将 {cart_features_temp} 写入第{index}行")
                self.x_in.loc[index, save_to_y] = cart_features_temp

    def _fractal_dimension(self):
        data = self.x.loc[:, self.all_boxs['select_feature_box'].get()]
        num_days = int(self.all_spins['frag_number'].get())
        end_datetime = self.all_spins['end_datetime'].get()
        feature = self.all_boxs['select_feature_box'].get()
        xs = cal_fd(data, num_days)
        fig = self.fig2 if self.canvas_select == 1 else self.fig1
        fig.clear()
        ax = fig.add_subplot(111)
        ax.plot(xs, label=feature)
        ax.axhline(y=1.25, ls='--', lw=1, c='r')
        ax.axvline(x=to_datetime(end_datetime), ls='--', lw=1, c='green')
        ax.set_title(end_datetime + 'frag_dimension:' + str(xs[to_datetime(end_datetime)])[:5])
        canvas = self.canvas2 if self.canvas_select == 1 else self.canvas1
        canvas.draw()
        self.auto_plot()

    def reduce(self):
        r_method = self.all_boxs['reduce_method'].get()
        y_chosen = self.all_boxs['y_select'].get()
        n_comp = 6
        methods = {'PCA': decomposition.PCA(n_components=n_comp),
                   'KernelPCA_sigmoid': decomposition.KernelPCA(n_components=n_comp, kernel="sigmoid"),
                   'KernelPCA_rbf': decomposition.KernelPCA(n_components=n_comp, kernel="rbf"),
                   'KernelPCA_poly': decomposition.KernelPCA(n_components=n_comp, kernel="poly"),
                   'TruncatedSVD': decomposition.TruncatedSVD(n_components=n_comp, algorithm='randomized'),
                   'FastICA': decomposition.FastICA(n_components=n_comp),
                   'Isomap': manifold.Isomap(n_neighbors=5, n_components=n_comp),
                   }
        y_data = self.y[y_chosen]
        validdate = Timestamp('2012-01-31')
        y_data = y_data.loc[(validdate < y_data.index)
                            & (y_data.index < Timestamp('2021-1-31'))]
        x_data = self.x.loc[y_data.index].fillna(method='bfill')
        features = x_data.columns
        min_max_scaler = preprocessing.MinMaxScaler(copy=False)
        x_data[features] = min_max_scaler.fit_transform(x_data[features])
        reducer = methods[r_method]
        reducer.fit(x_data[features])
        x_reduced = reducer.transform(x_data[features])
        x_reduced = DataFrame(x_reduced, index=x_data.index)
        # 拟合一个平面，该平面为理论上机器学习的上限，所有的点到平面距离之和最小。
        x_train = x_reduced
        y_train = y_data
        model = linear_model.BayesianRidge()
        model = model.fit(x_train, y_train)
        modelcoef = model.coef_
        modelintercept = model.intercept_
        planex0 = np.linspace(x_reduced[0].min(), x_reduced[0].max())
        planex1 = np.linspace(x_reduced[1].min(), x_reduced[1].max())
        planex0, planex1 = np.meshgrid(planex0, planex1)
        planey = modelcoef[0] * planex0 + modelcoef[1] * planex1 + model.intercept_  # 二元一次线性模型，绘制出平面
        y_pred = model.predict(x_train)
        rscore = metrics.r2_score(y_data, y_pred)
        # 绘图
        # %matplotlib qt
        fig1 = plt.figure(1, figsize=[12, 12])
        ax = Axes3D(fig1)
        if r_method in ['PCA', 'TruncatedSVD']:
            ax.set_xlabel(f"特征1, 占原信息{reducer.explained_variance_ratio_[0]:.2%}")
            ax.set_ylabel(f"特征2，占原信息{reducer.explained_variance_ratio_[1]:.2%}")
        else:
            ax.set_xlabel(f"特征1")
            ax.set_ylabel(f"特征2")
        ax.set_zlabel(f"{y_chosen}")
        ax.text2D(0.05, 0.95,
                  f"降维方法:{r_method}\n" +
                  r"线性回归的$r^{2}$:" + f"{rscore}\n",
                  fontsize='large', transform=ax.transAxes)
        ax.scatter(x_reduced[0], x_reduced[1], y_data, c=y_data, cmap='plasma', s=30)
        ax.plot_surface(planex0, planex1, planey, alpha=0.1)

        plt.figure(2, figsize=[12, 4])
        plt.plot(y_data.index, y_data, label='y real')
        plt.plot(y_data.index, y_pred, label='y regression')
        plt.title(f'BayesianRidge regression.')
        plt.legend()

    def askComboValue(self, *values):
        top = tk.Toplevel()  # use Toplevel() instead of Tk()
        tk.Label(top, text=f"将导出{self.cart_features_len}个特征到x_in_set_selector\n选择对应的y").pack()
        box_value = tk.StringVar()
        combo = tk.ttk.Combobox(top, textvariable=box_value, values=values)
        combo.pack()
        combo.bind('<<ComboboxSelected>>', lambda _: top.destroy())
        top.grab_set()
        top.wait_window(top)  # wait for itself destroyed, so like a modal dialog
        return box_value.get()


if __name__ == "__main__":
    ini_all_x_data()
    f = From()
    f.root.mainloop()
